package com.yxh.www.crawler.controller;

import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.loader.strategy.HtmlUnitPageLoader;
import com.xuxueli.crawler.parser.PageParser;
import com.yxh.www.crawler.client.ZmProxyIpCrawlerClient;
import com.yxh.www.crawler.parser.ZmProxyIpParser;
import com.yxh.www.crawler.util.CrawlerBuilder;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import java.util.Date;

/**
 * <p>
 * 测试爬虫
 * </p>
 *
 * @author yangxiaohui
 * @since 2020/5/15
 */
@Slf4j
public class TestXxlCrawler {

    void buildSimCrawler() {
        CrawlerBuilder.builder()
                .setUrls("http://myip.kkcha.com/")
                .setPageLoader(new HtmlUnitPageLoader())
                .setProxyMaker(new ZmProxyIpCrawlerClient().getRandomProxyMaker())
                .start(new PageParser<Object>() {
                    @Override
                    public void parse(Document document, Element element, Object o) {
                        log.info(document.text());
                    }
                }).stop();
    }
    public static void main(String[] args) {
        TestXxlCrawler testXxlCrawler = new TestXxlCrawler();
        testXxlCrawler.buildSimCrawler();
    }
}
